import importlib
import time
import six
from .signals import request_scheduler_popped
from scrapy.utils.misc import load_object

from .scrapy_base import defaults
from scrapy.http import Request


class Scheduler(object):
    def __init__(self, server=None,
                 persist=False,
                 flush_on_start=False,
                 queue_key=defaults.SCHEDULER_QUEUE_KEY,
                 queue_cls=defaults.SCHEDULER_QUEUE_CLASS,
                 dupefilter_key=defaults.SCHEDULER_DUPEFILTER_KEY,
                 dupefilter_cls=defaults.SCHEDULER_DUPEFILTER_CLASS,
                 idle_before_close=0,
                 serializer=None,
                 crawl_increment=False,
                 defer_time=45
                 ):
        if idle_before_close < 0:
            raise TypeError("idle_before_close cannot be negative")

        self.server = server
        self.persist = persist
        self.flush_on_start = flush_on_start
        self.queue_key = queue_key
        self.queue_cls = queue_cls
        self.dupefilter_cls = dupefilter_cls
        self.dupefilter_key = dupefilter_key
        self.idle_before_close = idle_before_close
        self.serializer = serializer
        self.stats = None
        self.crawl_increment = crawl_increment
        self.start_time = int(time.time())
        self.defer_time = defer_time

    def __len__(self):
        return len(self.queue)

    @classmethod
    def from_settings(cls, settings):
        kwargs = {
            'persist': settings.getbool('SCHEDULER_PERSIST'),
            'flush_on_start': settings.getbool('SCHEDULER_FLUSH_ON_START'),
            'idle_before_close': settings.getint('SCHEDULER_IDLE_BEFORE_CLOSE'),
        }

        optional = {
            'queue_key': 'SCHEDULER_QUEUE_KEY',
            'queue_cls': 'SCHEDULER_QUEUE_CLASS',
            'dupefilter_key': 'SCHEDULER_DUPEFILTER_KEY',
            'dupefilter_cls': 'DUPEFILTER_CLASS',
            'serializer': 'SCHEDULER_SERIALIZER',
        }
        for name, setting_name in optional.items():
            val = settings.get(setting_name)
            if val:
                kwargs[name] = val

        if isinstance(kwargs.get('serializer'), six.string_types):
            kwargs['serializer'] = importlib.import_module(kwargs['serializer'])

        kwargs["crawl_increment"] = settings.get('CRAWL_INCREMENT', True)
        kwargs["defer_time"] = settings.get('SYS_DEFER_TIME', 45)
        return cls(**kwargs)

    @classmethod
    def from_crawler(cls, crawler):
        instance = cls.from_settings(crawler.settings)

        instance.stats = crawler.stats
        instance.crawler = crawler
        return instance

    def open(self, spider):
        self.spider = spider
        self.server = self.spider.server
        try:
            self.queue = load_object(self.queue_cls)(
                server=self.server,
                spider=spider,
                key=self.queue_key % {'spider': f"{spider.name}{spider.serialNumber}"},
                serializer=self.serializer
            )
        except TypeError as e:
            raise ValueError(f"Failed to instantiate queue class {self.queue_cls}:{e}")

        try:
            self.df = load_object(self.dupefilter_cls)(
                server=self.server,
                key=self.dupefilter_key % {'spider': spider.name},
                debug=spider.settings.getbool('DUPEFILTER_DEBUG'),
            )
        except TypeError as e:
            raise ValueError(f"Failed to instantiate dupefilter class {self.dupefilter_cls}: {e}")

        if not self.crawl_increment or self.flush_on_start:
            spider.log('Clear {} request dup set'.format(self.dupefilter_key % {'spider': spider.name}))
            self.flush()
        if len(self.queue):
            spider.log(f"Resuming crawl ({len(self.queue)} requests scheduled)")

    def close(self, reason):
        pass

    def flush(self):
        self.df.clear()
        self.queue.clear()

    def enqueue_request(self, request):
        if request.meta.get("pop_filter") is True:
            fingerprint = request.meta.get("fingerprint")
            if fingerprint and isinstance(fingerprint, str):
                self.df.delete_df(fingerprint)
            return True
        include_headers = self.get_include_headers(request)
        if not request.dont_filter:
            s_urls = [s.url if issubclass(type(s), Request) else s for s in self.spider.start_urls]
            if self.df.request_seen(request, include_headers, s_urls):
                self.df.log(request, self.spider)
                return False
        if self.stats:
            self.stats.inc_value('scheduler/enqueued/db', spider=self.spider)
        if request.headers and "job_id" in request.headers:
            request.dont_filter = True
            request.headers.pop("job_id", "")
        if request and self.crawler and self.spider:
            self.crawler.signals.send_catch_log(signal=request_scheduler_popped,
                                                request=request, spider=self.spider)
        push_front = not request.dont_filter or request.meta.get("push_front") is True
        self.queue.push(request, push_front)
        return True

    def get_include_headers(self, request):
        include_headers = []
        if request.meta and request.meta.get("headers_4_dup"):
            headers = request.meta.get("headers_4_dup")
            if isinstance(headers, (list, tuple)):
                include_headers.extend(headers)
            elif isinstance(headers, str):
                include_headers.append(headers)
        return tuple(include_headers)

    def next_request(self):
        now = int(time.time())
        if now - self.start_time < self.defer_time:
            return
        block_pop_timeout = self.idle_before_close
        request = self.queue.pop(block_pop_timeout)
        if request and self.stats:
            self.stats.inc_value('scheduler/dequeued/db', spider=self.spider)
        if request and self.crawler and self.spider:
            self.crawler.signals.send_catch_log(signal=request_scheduler_popped,
                                                request=request, spider=self.spider)
        return request

    def has_pending_requests(self):
        return len(self) > 0
